package hasthi.tools.analyze;
import hasthi.common.HasthiException;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.StringReader;
import java.io.Writer;
import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static hasthi.tools.analyze.AnalyzeUtils.*;
import javax.xml.namespace.QName;

import org.apache.commons.math.stat.descriptive.DescriptiveStatistics;

import sun.nio.cs.ext.MacHebrew;

public class ParseHasthiData {
    private static HashMap<String, Experiment> dataMap = new HashMap<String, Experiment>();
    private static DecimalFormat df = new DecimalFormat(".0");
    
    private static String regex1 = "\\[([a-z]+).*?\\].*?faluire.*?Count=([0-9]+) DATA\\[(.*)\\]";
    private static Pattern pattern1 = Pattern.compile(regex1);
    
    //private static String regex2 = ".*?Manager \\(.*?\\).*?\\(([0-9]+)MB\\).*";
    private static String regex2 = ".*?Manager .*?\\(([0-9]+)MB\\).*";
    private static Pattern pattern2 = Pattern.compile(regex2);
//    private static String regex5 = ".*?Manager \\(.*?/(true|false)/([0-9]+)\\).*?\\(([0-9]+)MB\\)";
//    private static Pattern pattern5 = Pattern.compile(regex5);
    
    private static String regex3 = ".*?Coordinator.*?Mgrs.*?\\(([0-9]+)MB\\).*";
    private static Pattern pattern3 = Pattern.compile(regex3);
    
    private static String regex4 = "hasthi_([0-9]+)X([0-9]+)_.*?";
    private static Pattern pattern4 = Pattern.compile(regex4);
    

    //WARN [Timer-1]Manager (31128398/true/484/0) Fri Mar 28 22:13:19 EDT 2008(21MB)
    //[aoc(ADAS1206756802257Fri_Mar_28_22_13_22_EDT_2008)]10/10 mean =43.60 median=43.27 cov =.06 size =10 min=40.62 max =48.16 faluireCount=0 DATA[41.369 40.622 41.469 43.527 43.015 40.852 45.037 45.768 46.171 48.165 ]
    //WARN [Timer-2]Coordinator 1446 R->11 Mgrs Fri Mar 28 22:11:13 EDT 2008 (34MB)
    //hasthi_5000X11_03_28_22_09
    //[moc(mng6abb4e65-b1fa-4740-bcea-a4795e0e860b)]10/10 mean =16181.12 median=14040.31 cov =.89 size =10 min=7.84 max =42663.20 faluireCeCount=0 DATA[11.766 37.293 48.642 9170.816 13177.926 14912.719 25556.413 41209.47 28782.387 27833.368 ]
    public static void main1(String[] args) throws Exception {
////        System.out.println(testRegex(pattern5, "WARN [Timer-1]Manager (17241377/false/0) Thu Mar 27 17:35:37 EDT 2008(40MB)"));
////        System.out.println(testRegex(pattern3, "WARN [Timer-2]Coordinator 1446 R->11 Mgrs Fri Mar 28 22:11:13 EDT 2008 (34MB)"));
//        //System.out.println(testRegex(pattern5, "WARN [Timer-1]Manager (30641031/true/0) Thu Mar 27 17:36:32 EDT 2008(45MB)"));
//        //System.out.println(testRegex(pattern1, "[waitc]10/10 mean =72897.83 median=72894.87 cov =.64 size =10 min=6.50 max =145788.11 faluireCount=0 DATA[6.497 25789.274 30026.768 55788.934 60002.07 85787.678 90002.633 115788.162 119998.132 145788.109 ]"));
            ParseHasthiData hasthiData = new ParseHasthiData();
            hasthiData.parse(new File("/u/hperera/docs/drafts/plots/hasthi/mngrs_real_Sep18_08/rawdata"));
//        //hasthiData.parse(new File("/u/hperera/logs/hasthi_tmp"));
//        //hasthiData.parse(new File("/u/hperera/logs/hasthi_set3"));
//        hasthiData.parse(new File("/u/hperera/logs/hasthi_sim2"));
    }
    
    
    public static void main(String[] args) throws Exception {
        //args = new String[]{"/u/hperera/logs/mng1real_2008_Spet/"};
        ParseHasthiData hasthiData = new ParseHasthiData();
        String[] dirs = null;
        if(args.length == 0){
            dirs = new String[]{"."};
        }else{
            dirs = args;
        }
        
        for(String dir:dirs){
            final File file = new File(dir);
            if(new File(dir,"hasthi.log").exists()){
                hasthiData.processAresultDir(file);
                hasthiData.printResults(file);
            }else{
                hasthiData.parse(file);
            }
        }
    }
    
    
    
    
    

    public static boolean testRegex(Pattern pattern,String str){
        return pattern.matcher(str).matches();
    }
      

    public void parse(File baseDir) throws Exception{
        File[] dirlist = baseDir.listFiles();
        for(File file:dirlist){
            processAresultDir(file);
        }
        printResults(baseDir);
    }
    private void processAresultDir(File file) throws Exception {
        Matcher m = pattern4.matcher(file.getName());
        if(m.matches()){
//            int resourceCount = Integer.parseInt(m.group(1));
//            int managerCount = Integer.parseInt(m.group(2));
            String k1 = m.group(1);
            String k2 = m.group(2);
            String key = k2+"_"+k1;
            System.out.println(file.getName() +" = " + key);
            Experiment experiment = parseADir(file);
            dataMap.put(key, experiment);
            //break;
        }
    }
    
    public void printResults(File outputDir) throws Exception{
        
        File dataout = new File(outputDir,"dataout");
        dataout.mkdir();
        File summeryFile = new File(dataout,"summery.log");
        Writer w = new BufferedWriter(new FileWriter(summeryFile));
        Set<String> keys = new TreeSet<String>(new Comparator<String>() {
            public int compare(String o1, String o2) {
                String[] set1 = o1.split("_");
                String[] set2 = o2.split("_");
                
                if((set1.length == set2.length) && (set1.length == 2)){
                    int val1 = Integer.parseInt(set1[0]);
                    int val2 = Integer.parseInt(set2[0]);
                    if(val1 != val2){
                        return val1 - val2;
                    }else{
                        return Integer.parseInt(set1[1]) - Integer.parseInt(set2[1]);
                    }
                }
                return 0;
            }
        
        });
        keys.addAll(dataMap.keySet());
        for (String key : keys) {
            dataMap.get(key).processData();
        }

        
        Map<String, FormattedTable> dataByManagerCount = new HashMap<String, FormattedTable>();
        write(w,"========================== Results ================================");
        //write(w,"Key\t\tAgHB\t\tMLE2E\t\tCLE2E\t\tCLR\t\tMemMng\tMemCord\tMngHB");
        boolean isFirst = true;
        String header = null;
        
        FormattedTable table = null; 
        for(String key:keys){
            Experiment datastr = dataMap.get(key);
            if(isFirst){
                isFirst=false;
                header= datastr.getHeader();
                table = new FormattedTable(header);
            }
            
            String[] keyParts = key.split("_");
            if(key.length() < 7){
                key = key + "\t";
            }
            try {
                table.addLine(key+"\t"+datastr);
            } catch (Exception e) {
                throw new RuntimeException(datastr.getHeader() + " and \n "+ header + " does not match");
            }
            String managerCount = keyParts[0];
            String resourceCount = keyParts[1];
            
            FormattedTable tb = dataByManagerCount.get(managerCount);
            
            if(tb ==  null){
                tb = new FormattedTable(datastr.getHeader());
                dataByManagerCount.put(managerCount, tb);
            }
            tb.addLine(new StringBuffer().append(resourceCount).append("\t").append(datastr.set1()).toString());
        }
        write(w,table.toString());
        
        write(w,"===================================================================");
        for(String key:keys){
            write(w,dataMap.get(key).printdetails(key));
        }
        outputDir.mkdirs();
        for(String managerCount:dataByManagerCount.keySet()){
            FileWriter out = new FileWriter(new File(dataout,"mng"+managerCount+".data"));
            out.write(dataByManagerCount.get(managerCount).toString().replaceAll("[(]", "  ").replaceAll("[)]", ""));
            out.close();
        }
        
//        int[][] freqDatatCLE2E = new int[keys.size()][];
//        int[][] freqDatatMngHB = new int[keys.size()][];
//        int i = 0;
//        for(String key:keys){
//            Experiment exp = dataMap.get(key);
//            freqDatatCLE2E[i] = findFreqDistribution(exp.getData("CLE2E"));
//            freqDatatMngHB[i] = findFreqDistribution(exp.getData("MngHB"));
//            i++;
//        }    
//        writeFreqData(keys, freqDatatCLE2E, new File(outputDir,"CLE2E_freq.data").getAbsolutePath());
//        writeFreqData(keys, freqDatatMngHB, new File(outputDir,"MngHB.data").getAbsolutePath());
//        writePlotFile(keys, new File(outputDir,"/frq.plot").getAbsolutePath());
        
        //if(writeRawData){

//            for (String key : keys) {
//                Experiment exp = dataMap.get(key);
//                exp.writeRowData(new File(dataout,key+".data"));
//            }
        //}

        w.close();
        Experiment experiment = dataMap.get("3_26");
        DescriptiveStatistics data1 = experiment.getData("AgHB");
        //System.out.println(Arrays.toString(AnalyzeUtils.findFreqDistribution(data1)));
        System.out.println("AgHB\t"+ AnalyzeUtils.formatOne(AnalyzeUtils.removeOutliners(data1)));
        final DescriptiveStatistics data2 = experiment.getData("MngHB");
        //System.out.println(Arrays.toString(AnalyzeUtils.findFreqDistribution(data2)));
        System.out.println("MngHB\t" + AnalyzeUtils.formatOne(AnalyzeUtils.removeOutliners(data2)));
        DescriptiveStatistics data3 = experiment.getData("MLE2E");
        System.out.println("MLE2E\t"+ AnalyzeUtils.formatOne(AnalyzeUtils.removeOutliners(data3)));
        
    }
    
    public void write(Writer w,String str) throws IOException{
        System.out.println(str);
        w.write(str);
        w.write("\n");
    }
    
    public Experiment parseADir(File dir)throws Exception{
        Experiment experiment =  new Experiment();
        File[] files = dir.listFiles();
        for(int i = 0;i<files.length;i++){
            //if(files[i].getName().startsWith("action_corrdinator_")
            if(files[i].getName().startsWith("action_manager_")
                    ||files[i].getName().startsWith("hasthi_coordinator_")){
                parseDataFile(files[i], experiment,true);
            }else if(files[i].getName().startsWith("hasthi_manager_") 
                    || files[i].getName().startsWith("service")){
                parseDataFile(files[i], experiment,false);
            }else if(files[i].getName().startsWith("hasthi.log")){
                parseTraceFile(files[i], experiment);
            }
        }
        return experiment;
    }
    
    
    public void parseDataFile(File fileName, Experiment experiment,boolean coordinator) throws Exception{
        //This initialize everything
        experiment.getData("AgHB");experiment.getData("CLR");experiment.getData("MLR");
        experiment.getData("CLE2E");experiment.getData("MLE2E");experiment.getData("CoorRunc");
        experiment.getData("MRunc");experiment.getData("CoorWaitc");
        experiment.getData("MngHB");experiment.getData("MWaitc");experiment.getData("MemCord");
        
        BufferedReader br = new BufferedReader(new FileReader(fileName));
        String line = br.readLine();
        try {
            while (line != null) {
                line = line.trim();
                Matcher matcher = pattern1.matcher(line);
                if (matcher.matches()) {
                    String name = matcher.group(1);
                    int faluireCount = Integer.parseInt(matcher.group(2));
                    String[] dataValues = matcher.group(3).split(" ");
                    DescriptiveStatistics data;
                    if(name.equals("aoc")){
                        data = experiment.getData("AgHB");
                    }else if(name.equals("roc")){
                        data = coordinator?experiment.getData("CLR"):experiment.getData("MLR");
                    }else if(name.equals("loc")){
                        data = coordinator?experiment.getData("CLE2E"):experiment.getData("MLE2E");
                    }else if(name.equals("runc")){
                        data = coordinator?experiment.getData("CoorRunc"):experiment.getData("MRunc");
                    }else if(name.equals("waitc")){
                        data = coordinator?experiment.getData("CoorWaitc"):experiment.getData("MWaitc");
                    }else if(name.equals("moc")){
                        data = experiment.getData("MngHB");
                    }else{
                        throw new Exception("Unknown name "+name);
                    }
                    for(String val:dataValues){
                        data.addValue(Double.parseDouble(val));
                    }
                    
                } else{
                    if(!line.startsWith("#") && !line.startsWith("[History]:")){
                        System.out.println("does not match "+line);    
                    }
                    
                }   
                line = br.readLine();
            }
           
        } catch (Throwable e) {
            //throw new Exception("Error at "+ line,e);
            System.out.println("Error at "+ line + e.getMessage());
        }finally{
            br.close();
        }
    }
    
    public void parseTraceFile(File fileName, Experiment experiment)throws Exception{
        BufferedReader br = new BufferedReader(new FileReader(fileName));
        String line = br.readLine();
        while (line != null) {
            line = line.trim();
            Matcher matcher1 = pattern2.matcher(line);
            
            
            int  memeory = -1;
            //int resourceCount = -1;
            if (matcher1.matches()) {
                memeory = Integer.parseInt(matcher1.group(1));
//                resourceCount = Integer.parseInt(matcher1.group(1));
//                if(resourceCount > 10){
//                    memeory = Integer.parseInt(matcher1.group(3));
//                }
            }
            if(memeory > 0){
                experiment.addDatapoint("MemMng",memeory);
            }else{
                Matcher matcher2 = pattern3.matcher(line);
                if(matcher2.matches()){
                    //resourceCount = Integer.parseInt(matcher2.group(1));
                    memeory = Integer.parseInt(matcher2.group(1));
                    experiment.addDatapoint("MemCord",memeory);
                }else{
                    if(!line.startsWith("#") && (line.contains("Manager ")||line.contains("Coordinator "))){
                        System.out.println("Can not handle "+line);
                    }
                }
            }
           
            line = br.readLine();
        }
        br.close();
    }
    

//    public static class Experiment {
//        DescriptiveStatistics agentHeartbeat = DescriptiveStatistics.newInstance();
//        DescriptiveStatistics managmentLoopE2e= DescriptiveStatistics.newInstance();
//        DescriptiveStatistics managmentLoopRules= DescriptiveStatistics.newInstance();
//        DescriptiveStatistics coordinatorLoopE2e= DescriptiveStatistics.newInstance();
//        DescriptiveStatistics coordinatorLoopRules= DescriptiveStatistics.newInstance();
//        DescriptiveStatistics coordinatorActonsExec= DescriptiveStatistics.newInstance();
//        DescriptiveStatistics coordinatorActonsWait= DescriptiveStatistics.newInstance();
//        DescriptiveStatistics managerActionsExec= DescriptiveStatistics.newInstance();
//        DescriptiveStatistics managerActionsWait= DescriptiveStatistics.newInstance();
//        DescriptiveStatistics memoryManager= DescriptiveStatistics.newInstance();
//        DescriptiveStatistics memeoryCoordinator= DescriptiveStatistics.newInstance();
//        DescriptiveStatistics managerHeartbeat= DescriptiveStatistics.newInstance();
//        
//        public String printdetails(String name){
//            StringBuffer buf = new StringBuffer();
//            buf.append("### ").append(name).append(" ####\n");
//            buf.append("Name                  \tmean\tmedian\tcov\tsize\tmin\tmax\n");
//            
//            buf.append("agentHeartbeat       \t").append(formatOne(agentHeartbeat));
//            buf.append("managmentLoopE2e     \t").append(formatOne(managmentLoopE2e));
//            buf.append("managmentLoopRules   \t").append(formatOne(managmentLoopRules));
//            buf.append("coordinatorLoopE2e   \t").append(formatOne(coordinatorLoopE2e));
//            buf.append("coordinatorLoopRules \t").append(formatOne(coordinatorLoopRules));
//            buf.append("coordinatorActonsExec\t").append(formatOne(coordinatorActonsExec));
//            buf.append("coordinatorActonsWait\t").append(formatOne(coordinatorActonsWait));
//            buf.append("managerActionsExec   \t").append(formatOne(managerActionsExec));
//            buf.append("managerActionsWait   \t").append(formatOne(managerActionsWait));
//            buf.append("memoryManager        \t").append(formatOne(memoryManager));
//            buf.append("memeoryCoordinator   \t").append(formatOne(memeoryCoordinator));
//            buf.append("managerHeartbeat     \t").append(formatOne(managerHeartbeat));
//            return buf.toString();
//        }
//        
//        public void dataPoint(StringBuffer buf,DescriptiveStatistics data){
//            buf.append(df.format(data.getMean())).append("(").append(df.format(confidence95Interval(data))).append(")\t");
//        }
//      
//        
//        public String toString(){
//            StringBuffer buf = new StringBuffer();
//            
//            dataPoint(buf,agentHeartbeat);
//            dataPoint(buf,managmentLoopE2e);
//            //dataPoint(buf,managmentLoopRules);
//            dataPoint(buf,coordinatorLoopE2e);
//            dataPoint(buf,coordinatorLoopRules);
//            //dataPoint(buf,coordinatorActonsExec);
//            //dataPoint(buf,coordinatorActonsWait);
//            //dataPoint(buf,managerActionsExec);
//            //dataPoint(buf,managerActionsWait);
//            buf.append(df.format(memoryManager.getMax())).append("\t");
//            buf.append(df.format(memoryManager.getMax())).append("\t");
//            dataPoint(buf,managerHeartbeat);
//
//            return buf.toString();
//        }
//        
//        
//        
//        
//        public String set1(){
//            StringBuffer buf = new StringBuffer();
//            buf.append(df.format(coordinatorLoopE2e.getMean())).append("\t");
//            buf.append(df.format(managerHeartbeat.getMean())).append("\t");
//            buf.append(df.format(memoryManager.getMax())).append("\t");
//            buf.append(df.format(memeoryCoordinator.getMax())).append("\t");
//            buf.append(df.format(agentHeartbeat.getMean())).append("\t");
//            buf.append(df.format(managmentLoopE2e.getMean())).append("\t");
//            
//            
//            buf.append(df.format(confidence95Interval(coordinatorLoopE2e))).append("\t");
//            buf.append(df.format(confidence95Interval(managerHeartbeat))).append("\t");
//            buf.append(df.format(confidence95Interval(agentHeartbeat))).append("\t");
//            buf.append(df.format(confidence95Interval(managmentLoopE2e))).append("\t");
//            
//            return buf.toString();
//        }
//
//        
//    }
//    
//    public void writePlotFile(Set<String> keys,String file) throws IOException{
//        FileWriter writer = new FileWriter(file);
//        writer.write("set terminal png\n");
//        writer.write("set output \"frqCLE2E.png\"\n");
//        writer.write("set title \"Frequnecy vs. Steps\"\n");;
//        writer.write("set ylabel \"Frequnecy\"\n");
//        writer.write("set xlabel \"Step\"\n");
//        writer.write("set key right top\n");
//        writer.write("plot "); 
//        int i = 0;
//        for(String key:keys){
//            writer.write("\t\"CLE2E_freq.data\" using 1:"+(i+2)+" title \""+key+"\" with linespoints");
//            if(i != keys.size()-1){
//                writer.write(", \\");         
//            }
//            writer.write("\n");
//            i++;
//        }
//        
//        writer.write("set output \"frqMngHB.png\"\n");
//        writer.write("plot ");
//        i = 0;
//        for(String key:keys){
//            writer.write("\t\"MngHB.data\" using 1:"+(i+2)+" title \""+key+"\" with linespoints");
//            if(i != keys.size()-1){
//                writer.write(", \\");         
//            }
//            writer.write("\n");
//            i++;
//        }
//        writer.close();
//        
//    }
//    
//    
//    public void writeFreqData(Set<String> keys,int[][] data,String file) throws IOException{
//        FileWriter writer = new FileWriter(file);
//        writer.write("#");
//        for(String name:keys){
//            writer.write(format(name));
//        }
//        writer.write("\n");
//        int expCount = data.length;
//        int freDataCount = data[0].length;
//        for(int i=0;i<freDataCount;i++){
//            writer.write(format(String.valueOf(i)));
//            for(int j=0;j<expCount;j++){
//                writer.write(format(String.valueOf(data[j][i])));
//            }
//            writer.write("\n");
//        }
//        writer.close();
//        
//    }
    
    

}